{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec113f8e-56c1-44f7-9571-292c946bb6f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob, json, os, sys\n",
    "\n",
    "sys.path.append('/path/to/metrics/') # TODO: Replace with path to `SWE-bench/metrics` folder\n",
    "from conversion import convert_log_to_ground_truth\n",
    "from getters import get_logs_gold\n",
    "from monitor import monitor_validation, monitor_logs_same_diff\n",
    "sys.path = sys.path[:-1]\n",
    "\n",
    "sys.path.append('/path/to/harness/') # TODO: Replace with path to `SWE-bench/harness` folder\n",
    "from utils import has_attribute_or_import_error\n",
    "sys.path = sys.path[:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64e965b7-702e-49d4-bc15-2619dc185752",
   "metadata": {},
   "source": [
    "Declare repository; Fetch tasks, logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f1e9497-6e60-4027-8cee-790c8d0f1886",
   "metadata": {},
   "outputs": [],
   "source": [
    "repo = 'pydicom/pydicom' # TODO: Replace with repository name\n",
    "log_dir = '/logs/validation/pydicom' # TODO: Replace with path to folder of execution logs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bba9ae3b-726c-40ec-8ed2-4012fc4f70dc",
   "metadata": {},
   "source": [
    "Get map of version to setup commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daa9d915-da85-4ca0-bba1-cd61ffeef3c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = json.load(open('path/to/pydicom-task-instances.json')) # TODO: Replace with path to versioned candidate task instances\n",
    "tasks = sorted(tasks, key=lambda x: x['created_at'], reverse=True)\n",
    "version_to_setup_commit = {}\n",
    "for t in tasks:\n",
    "    if 'version' in t and t['version'] not in version_to_setup_commit:\n",
    "        version_to_setup_commit[t['version']] = t['base_commit']\n",
    "assert(\n",
    "    sorted(list([x or \"\" for x in version_to_setup_commit.keys()])) ==\n",
    "    sorted(list(set([t['version'] or \"\" for t in tasks if 'version' in t])))\n",
    ")\n",
    "tasks = {t['instance_id']: t for t in tasks}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0994a4a9-fd3d-4554-90b7-24b10c499b8a",
   "metadata": {},
   "source": [
    "#### Monitor Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5fcd54c-1a45-41f9-8a9c-64e9ff9df31a",
   "metadata": {},
   "outputs": [],
   "source": [
    "failed_install, corrupt_test_patch, corrupt_patch, timeout, success = monitor_validation(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecf576b3-22ed-4d8c-a40d-340ae8c56db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "logs_same, logs_diff = monitor_logs_same_diff(log_dir)\n",
    "print(f\"Logs same: {len(logs_same)}\")\n",
    "print(f\"Logs diff: {len(logs_diff)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b81a74a5-c56e-4724-acf5-378b601a2d89",
   "metadata": {},
   "source": [
    "#### Get [FP]2[FP] Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb935466-43e8-45a7-8f46-15925d82312b",
   "metadata": {},
   "outputs": [],
   "source": [
    "inst_id_to_gt = {}\n",
    "for d in logs_diff:\n",
    "    status_gt = convert_log_to_ground_truth(d[0])\n",
    "    inst_id_to_gt[d[0]] = status_gt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccaa7863-f144-49da-ae92-6430841bd3ea",
   "metadata": {},
   "source": [
    "#### Create Task Instances `.json` file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44a4dc43-d158-4b3e-b7cd-728954bce188",
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks_final = []\n",
    "get_id_from_log = lambda x: x.split('/')[-1].split('.')[0]\n",
    "for k, v in inst_id_to_gt.items():\n",
    "    if len(v['FAIL_TO_PASS']) == 0:\n",
    "        continue\n",
    "    task = tasks[get_id_from_log(k)]\n",
    "    task['FAIL_TO_PASS'] = v['FAIL_TO_PASS']\n",
    "    task['PASS_TO_PASS'] = v['PASS_TO_PASS']\n",
    "    task['environment_setup_commit'] = version_to_setup_commit[task['version']]\n",
    "\n",
    "    # Do not consider tasks where the log before the patch has an attribute/import error\n",
    "    log_path = os.path.join(log_dir, f'{t[\"repo\"].split(\"/\")[-1]}/{t[\"instance_id\"]}.log')\n",
    "    log_before, log_after = get_logs_gold(log_path)\n",
    "    if has_attribute_or_import_error(log_before):\n",
    "        continue\n",
    "\n",
    "    tasks_final.append(task)\n",
    "print(f\"Final number of task instances: \", len(tasks_final))\n",
    "\n",
    "SAVE_PATH = \"path/to/save/tasks/to.json\" # TODO: Replace this with a path to a .json file to save the task instances\n",
    "with open(SAVE_PATH, 'w') as f:\n",
    "    json.dump(tasks_final, fp=f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
